{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prompt Optimization\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-07-22 04:29:16.596\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mprompt\u001b[0m:\u001b[36mload_prompt\u001b[0m:\u001b[36m281\u001b[0m - \u001b[1mLoading prompt from prompts/optimization/apply_gradient.md\u001b[0m\n",
      "\u001b[32m2024-07-22 04:29:16.604\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mprompt\u001b[0m:\u001b[36mload_prompt\u001b[0m:\u001b[36m281\u001b[0m - \u001b[1mLoading prompt from prompts/optimization/cleanup_text.md\u001b[0m\n",
      "\u001b[32m2024-07-22 04:29:16.604\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mprompt\u001b[0m:\u001b[36mload_prompt\u001b[0m:\u001b[36m281\u001b[0m - \u001b[1mLoading prompt from prompts/optimization/cleanup_text.md\u001b[0m\n",
      "\u001b[32m2024-07-22 04:29:16.611\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mprompt\u001b[0m:\u001b[36mload_prompt\u001b[0m:\u001b[36m281\u001b[0m - \u001b[1mLoading prompt from prompts/optimization/prompt_loss.md\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from textwrap import dedent\n",
    "\n",
    "from tinytextgrad import optimize_prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-07-22 04:30:20.936\u001b[0m | \u001b[34m\u001b[1mDEBUG   \u001b[0m | \u001b[36mtinytextgrad\u001b[0m:\u001b[36moptimize_text\u001b[0m:\u001b[36m291\u001b[0m - \u001b[34m\u001b[1m∇ 1. Current prompt:\n",
      "\n",
      "Given some text and a question, determine if the text\n",
      "contains the answer to the question\u001b[0m\n",
      "\u001b[32m2024-07-22 04:31:20.375\u001b[0m | \u001b[34m\u001b[1mDEBUG   \u001b[0m | \u001b[36mtinytextgrad\u001b[0m:\u001b[36mforward\u001b[0m:\u001b[36m220\u001b[0m - \u001b[34m\u001b[1m∇ Evaluation input:\n",
      "\n",
      "Text:\n",
      "Given some text and a question, determine if the text\n",
      "contains the answer to the question\n",
      "\n",
      "Results:\n",
      "Input: Text: Shakespeare wrote 'Romeo and Juliet.' Question: Who wrote 'Hamlet'?\n",
      "Output:  if and Userlet question and and toietomecontains questionleto ifoam answeristantlet and ' and Whoam' answeram answer:##letText\n",
      "Input: Text: Coffee is typically grown in tropical regions. Question: Where is coffee typically grown?\n",
      "Output: off the## ### to## andoff isText ###Text typicallyistant and to tropical and in is C if. grown and grown typicallyee answer in if typically is and to grown is theoff Questionee Question to if in C in ### typically Question: Question if is if:istantoff to: typically the the is. answer is C ### Question the Question tropical. is: C typically tropical is C the if typically the answer typically answer is is C.off Question the the and tooff typically if and.istant tropical: Question answer tropical tooff Question ###off answer andoff Question:: the Question the typically answeristant Question Question typically typically if tropical ###off answer toistant typically is typically. ### tropical: Question the...istantoff. if to C and ### if Question if ### answer typically ###. is ### C is answer answer typically andistant if typically is tropical to typically: Question answer typically ### typicallyistantoff.istantistant isoff and ### ###istant is Question ### the tropical and tropical Cistant the the typically C and ifoff typically: is to the Questionistant Question answer. ###. answer ### typically answer.off answer:off tropical C: to if the if tropical typically typically ###:off Cistant if Cistant tropical:off is to. Question to if the tropical and C the is Question and answer if Questionoff. typically Coff is tropical Question answer to and answer Coff ### and is tropical tropical the C to the ###.off tropical to if ### C if Question and is to answer: answer tropical: to if ### ###: and to is C.. the: typically ### answer:off and ### is the answer the is. and. ifoff the to C is to C. Question if tropicaloff if Question C C the typically. tropical. and. is ### Question if tropical to Question to the typically C C answer and and: typicallyoff if. to if answer typically C ifoff typically typically tropical and typically: the: if if to if C typically ### to:off: is: C to tropical the answer::offoff ###: the Question Question tropical Question answer if the Question tropicaloff if:. typically if.off tropical the Question tropical. ### and the C ### ### Question is: and. ifoffoff the: C if ### is to is is and tropicaloff ### the the and typically ### the andoff answer the tropical C Question and is C C: typically typically ### C tropical the:off is is if is theoff is typically if and to tropical if C the is tooff:off: and ### ifoff if the is Question answer is is is if to tropical and the to to Question C Question tropical is the ### typically the Question tropical Question is answer C if to if andoff and to:off to answer theoff C to is and Question ###off Questionoff tropical ### to tropicaloff: C the tropical ### answer typically andoffoff the and tropical if and tropical to the answer if is C C is answer: is typically if and and: answer tropical Coff the to is answer Question: and typically Questionoff answer and C is Question typically typically ### and and ### Coff Question the to if: Question C to typically answer the to: ###off to typically C C is typically C and ifoff answer and Question theoffoff the if the the isoff tropical typically andoff C and: if is ### C the is answer C is C and and is tropical tropicaloff the is tropical if C tropical the tropicaloff theoff Question typically and is Question answer typicallyoff answer Coff and typically C typically is if ### typically Question the typically ### the ### the Question Question if ### the ### C isoff and tropical answer ### the tropical tropical typically the C and answer and ### if typically if tropical ifoff if the typically answer tropical and tropical and is answer the typically Question answer tropical ### C tropical if ###off Question Question answer if Question tropicaloff answer the if ### typically and if answer the Question answeroff and if typically and C is answer C is if typically C and Question ifoff answeroff tropicaloff andoff if typically Question and answer C ### Question Coff ifoff answer is ### Question C is if C and and ### C and typically and if Question Question if tropicaloffoff and if the ### if the the answer answer typically Question Question C is andoffoff if is isoff C C and and tropical typicallyoff Question tropical answer is tropical ifoff Question typically if is is theoff ### tropical the Question is C Question and C is Question if typicallyoff ### Question answer Question typicallyoff answer if answer and if tropical is is is tropical C if the the tropical Questionoff is tropical typically and answer Question and typically andoff ### ### Question typically tropical the tropical if the and is is ### answer and answer tropical tropicaloff tropical Question andoff and ### the ifoff is C the ###off andoff ### and typically answer typically C answeroff typically the and typically typically typically tropical the if if typically C ### is tropicaloff typicallyoff typically is and is and and Coff typically tropical if answer tropical the ### typically answer tropical Question is answer is Question typically if is if if C typically the answer Question C is Coff is typically answer if answer Question and and is Question and C C ### tropical C Question is and typically answer Question ### Question tropical ### if ### typically and answer Question Question typically typically typically Questionoff is if Coff ###off C C if the the and the and C typicallyoff typically ### is if answer ### and typically answer the tropicaloff ### Question ### tropical typically the typically if typically if Question ifoff answer ###offoff tropical typically answer answer C the ### ### the typically typically the C is and tropical is tropical typically and ### and isoff typically Question answer C Question ### Question C Question ### the typicallyoff C the if if is ### answer and is tropicaloff the C C tropical typically isoff tropical if typically ### answeroff C tropical tropical answer Questionoff answeroff tropical Question Question if Question typically tropical ### tropicaloff Question theoff the the and typically tropical the and the and tropicaloff is tropical tropical is isoff tropical theoff the Question if typically if C the typically answer Question if isoffoff and and and tropical Coff Question Question ### and tropical Question Question the is tropical the ifoff the if is and is is ### the typically C tropical Question tropical the is Questionoff andoff tropical and tropical Coff is the and C the is answer typically the answer tropical is typically the the ### and answer Question typically is tropical Questionoff if ### typically the and if tropical ### tropical Question Question if the answer C typically and if ### tropical C the ifoff Coff answer is if and Question Question the the ifoff ### and answeroff is and C answer and answer the tropicaloff ### C the if and and typically the ### Question typically Question typically C C if Questionoff tropical ### the Question ### ### answer answer typically typically tropical tropical C tropical answer and and answeroffoff if ### and Question if ### the ### and is answer answer C tropical if ### tropicaloff is Question the is ### typically if ### tropical typically tropical tropical and ifoff and ###off ### if the and C typicallyoff C typically if C Coff C answeroff Coff C ### is answer the answer andoffoff is is answer typicallyoff ### is the is is C C C ### is tropicaloffoff typically C ### C and answer the typically tropicaloff ### and answer answer ### typically if C answer the andoff if andoff is tropicaloff is and if and typically the tropical ### isoff answer and ### answeroff answer answer ### is is ### is is answer typically the Coff C the ### the and if the typically C typically ###off is and ### ### answer C and tropical if tropicaloff if typically if C typically ### if is C C ### answer and if ### is C theoff tropical ### the and tropical if is typically ### is answeroff C ### and and ### is typicallyoff is if ### if answer is is C typically the and is answeroff tropical answer if tropical answer and typically ### if if if and ### typically ###off typically the typicallyoff typically C typically if if answer C C Coff is Coffoffoffoff ### tropical tropical C C ###offoff the ### typically if typicallyoff ### typically ### if tropical typically the typically ### if theoff and C and answer C is answer answer answer answer C the tropical C ### the typically and typically ### C ### ifoff typically ### the ### tropical typically tropical tropical the ### if tropical and and and the is answer is tropical tropical if answer if ### and answer typically and the ### typically and and andoff typically answer the is typically tropical answer answer answer answer theoff C C tropical and the tropical C is typically the the tropical tropical ### ###off ### typically and typically C the is if ifoff and the typically answer is and tropicaloff and ### typically and ###off the the C the ###off isoff Coff ifoff is typically answer isoff tropical typically if the and typically the if typically if is answer the ### is answer if tropical tropical ### answer answer ### if tropical is and is ### tropical if if Coff tropical typically C if answer ### ### typically answer C if typically typically the and typically C ### the if is C the tropical C tropicaloffoff ### if answer tropical C ### the answer ### if if ### the and tropical is ###off ### the answer typically C the C theoff answer typically is answeroff C C C C typicallyoff C answer typically the the tropical is C C the and typically tropical tropical answer the typically answer tropical ### if if and the ### ### tropical typically if and C answer ### and the ### C the answer ifoff is is is isoff answer typically ### answer ###off answer tropical ### the answer answer is is is answer typically theoff C ###off is answer answer C tropical tropical if tropical typically is ### answer ### the typically the and C ###off ### if and the typically typically answer typically typically and tropical is is Coff and tropicaloff ###off answer tropical typically ### C theoff tropical ### is the and ###offoff and and C C if typicallyoff the and if if if C if is if C\n",
      "Input: Text: The Earth orbits the Sun. Question: What does the Earth orbit?\n",
      "Output:  text<unk> if Question Sun The## question if Ass<unk> a if the question orText or or Ass if a Ass# does. The the Question does question does\n",
      "Input: Text: The sky is blue. Question: What color is the grass?\n",
      "Output: Text<unk> The Ass is Question and if someText Ass User Ass, some Question ifText and grass\n",
      "Input: Text: Paris is the capital of France. Question: What is the capital of Germany?\n",
      "Output:  Question Question determine answerText is question Ass####\u001b[0m\n",
      "\u001b[32m2024-07-22 04:31:28.391\u001b[0m | \u001b[34m\u001b[1mDEBUG   \u001b[0m | \u001b[36mtinytextgrad\u001b[0m:\u001b[36mstep\u001b[0m:\u001b[36m268\u001b[0m - \u001b[34m\u001b[1m∇ Loss:\n",
      "\n",
      "### Evaluation of the Prompt and Results\n",
      "\n",
      "**Clarity:**  \n",
      "The prompt lacks clarity. It does not clearly define how the AI should respond when the answer is not found in the text, leading to confusion in the outputs. Phrasing such as \"determine if the text contains the answer\" could be more straightforward.\n",
      "\n",
      "**Specificity:**  \n",
      "The prompt is somewhat specific in indicating that the AI should check if the text contains an answer to the question. However, it does not specify what to do in cases where the answer is absent, which is crucial for guiding the AI's response.\n",
      "\n",
      "**Relevance:**  \n",
      "The information in the prompt is generally relevant to the task of determining if a text contains the answer to a question. However, it could benefit from additional context about how to handle cases where the answer is not present in the text.\n",
      "\n",
      "**Consistency:**  \n",
      "There are inconsistencies in the outputs, as they do not uniformly follow a pattern or provide coherent responses. Some responses are fragmented or nonsensical, indicating a lack of consistent processing by the AI.\n",
      "\n",
      "**Actionability:**  \n",
      "The prompt does not effectively guide the AI to produce a concrete output when the answer is not found. The AI should be instructed to indicate when information is missing and potentially provide related information or suggest sources for the answer.\n",
      "\n",
      "**Output Quality:**  \n",
      "The results are poorly constructed, often resulting in nonsensical or irrelevant responses. They do not effectively address the prompt's requirements, particularly when the answer is not present in the text.\n",
      "\n",
      "**Output Consistency:**  \n",
      "Multiple runs produce varied and inconsistent outputs, which do not adhere to a standard format or logic. This inconsistency undermines the reliability of the responses.\n",
      "\n",
      "**Efficiency:**  \n",
      "The prompt does not achieve the desired outcome efficiently. The complexity and lack of clarity in instructions lead to convoluted outputs rather than straightforward responses.\n",
      "\n",
      "### Handling of Cases Where the Answer is Not in the Text\n",
      "\n",
      "The prompt does not adequately handle cases where the answer is not found in the text. A good prompt should:\n",
      "\n",
      "- Encourage the AI to clearly state when information is unavailable.\n",
      "- Guide the AI to provide related information or suggest where to find the answer.\n",
      "- Avoid fabricating information to fill gaps.\n",
      "- Allow for clarification requests when necessary.\n",
      "\n",
      "### Suggestions for Improvement\n",
      "\n",
      "1. **Rewording for Clarity:**  \n",
      "   The prompt should be rephrased to clearly state that the AI needs to check for the answer and indicate what to do if it is not found (e.g., \"Please check if the text contains the answer to the question. If not, state that the answer is not available and suggest relevant information or sources.\").\n",
      "\n",
      "2. **Adding Specificity:**  \n",
      "   Include explicit instructions for handling situations where the answer is absent. This could involve providing an example of how to respond in such cases.\n",
      "\n",
      "3. **Restructuring the Prompt:**  \n",
      "   A more structured prompt that explicitly separates the process of checking the text from the response strategy could lead to better outputs.\n",
      "\n",
      "4. **Incorporating Guidelines for Output Quality:**  \n",
      "   Incorporate guidelines that require the AI to produce coherent and relevant outputs, with a focus on clarity and completeness.\n",
      "\n",
      "5. **Making the Prompt More Robust:**  \n",
      "   To improve robustness, include examples of both successful and unsuccessful queries. This can help the AI learn how to respond appropriately across different scenarios.\n",
      "\n",
      "By implementing these suggestions, the prompt could be refined to guide the AI more effectively, leading to clearer, more relevant, and consistent outputs.\u001b[0m\n",
      "\u001b[32m2024-07-22 04:31:31.517\u001b[0m | \u001b[34m\u001b[1mDEBUG   \u001b[0m | \u001b[36mtinytextgrad\u001b[0m:\u001b[36mbackward\u001b[0m:\u001b[36m173\u001b[0m - \u001b[34m\u001b[1m∇ Updated value:\n",
      "\n",
      "Given a text and a question, please check if the text contains the answer to the question. If the answer is present, provide it. If the answer is not available, clearly state that the answer is not found in the text and suggest related information or potential sources where the answer might be located.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Final optimized TEXT_CONTAINS_ANSWER_PROMPT:\n",
      "Given a text and a question, please check if the text contains the answer to the question. If the answer is present, provide it. If the answer is not available, clearly state that the answer is not found in the text and suggest related information or potential sources where the answer might be located.\n"
     ]
    }
   ],
   "source": [
    "initial_prompt = dedent(\"\"\"\n",
    "Given some text and a question, determine if the text\n",
    "contains the answer to the question\n",
    "\"\"\").strip()\n",
    "\n",
    "data = [\n",
    "  \"Text: The cat is on the mat. Question: Where is the cat?\",\n",
    "  \"Text: The sky is blue. Question: What color is the grass?\",\n",
    "  \"Text: Paris is the capital of France. Question: What is the capital of Germany?\",\n",
    "  \"Text: The Earth orbits the Sun. Question: What does the Earth orbit?\",\n",
    "  \"Text: Water freezes at 0 degrees Celsius. Question: At what temperature does water boil?\",\n",
    "  \"Text: Beethoven composed many symphonies. Question: Who composed the Fifth Symphony?\",\n",
    "  \"Text: Elephants are the largest land animals. Question: What is the largest land animal?\",\n",
    "  \"Text: Shakespeare wrote 'Romeo and Juliet.' Question: Who wrote 'Hamlet'?\",\n",
    "  \"Text: Humans have 206 bones. Question: How many bones do humans have?\",\n",
    "  \"Text: Coffee is typically grown in tropical regions. Question: Where is coffee typically grown?\",\n",
    "]\n",
    "inputs = random.sample(data, 5)\n",
    "\n",
    "result = optimize_prompt(\n",
    "  initial_prompt=initial_prompt,\n",
    "  prompt_inputs=inputs,\n",
    "  num_iterations=1,\n",
    ")\n",
    "\n",
    "print(\n",
    "  \"\\n\\nFinal optimized TEXT_CONTAINS_ANSWER_PROMPT:\"\n",
    ")\n",
    "print(result)\n",
    "\n",
    "TEXT_CONTAINS_ANSWER_PROMPT = result.to_prompt()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
